{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import os\n",
    "from datetime import datetime\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm\n",
    "\n",
    "from moment.common import PATHS\n",
    "from moment.utils.ucr_anomaly_archive_subset import ucr_anomaly_archive_subset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "EXPERIMENT_NAME = \"supervised_anomaly_detection\"\n",
    "\n",
    "def get_model_results(model_name):\n",
    "    if model_name == 'MOMENT_LP':\n",
    "        # results_path = os.path.join(\n",
    "        #     PATHS.RESULTS_DIR, EXPERIMENT_NAME, model_name, 'linear-probing')\n",
    "        results_path = os.path.join(\n",
    "            '/home/scratch/mgoswami/moment_results/', EXPERIMENT_NAME, 'MOMENT', 'linear-probing')\n",
    "    if model_name == 'MOMENT_0':\n",
    "        # results_path = os.path.join(\n",
    "        #     PATHS.RESULTS_DIR, EXPERIMENT_NAME, model_name, 'linear-probing')\n",
    "        results_path = os.path.join(\n",
    "            '/home/scratch/mgoswami/moment_results/', \"zero_shot_anomaly_detection\")\n",
    "    elif model_name in ['DGHL', 'AnomalyTransformer', 'TimesNet', 'GPT4TS', 'AnomalyNearestNeighbors']:\n",
    "        # results_path = os.path.join(\n",
    "        #     PATHS.RESULTS_DIR, EXPERIMENT_NAME, model_name, 'end-to-end')\n",
    "        results_path = os.path.join(\n",
    "            \"/home/scratch/mgoswami/moment_results/\", EXPERIMENT_NAME, model_name, 'end-to-end')\n",
    "\n",
    "    datasets_without_results = []\n",
    "    results = []\n",
    "    for dataset_name in tqdm(ucr_anomaly_archive_subset, total=len(ucr_anomaly_archive_subset)): \n",
    "        metadata = dataset_name.split('/')[-1].split('_')\n",
    "        data_id, data_name = metadata[0], metadata[3]\n",
    "\n",
    "        if not os.path.exists(os.path.join(results_path, f\"results_{data_id}_{data_name}.csv\")):\n",
    "            datasets_without_results.append(dataset_name)\n",
    "            print(f\"Missing results for {os.path.join(results_path, f'results_{data_id}_{data_name}.csv')}\")\n",
    "            continue\n",
    "        \n",
    "        best_result_path = os.path.join(results_path, f\"results_{data_id}_{data_name}.csv\")\n",
    "        df = pd.read_csv(best_result_path, index_col=0).T\n",
    "\n",
    "        if 'Model name' in df.columns:\n",
    "            df['Run name'] = df['Model name']\n",
    "        else: \n",
    "            df['Run name'] = f'MOMENT_0'\n",
    "            df['ID'] = 0\n",
    "        df['Model name'] = model_name\n",
    "        df['Dataset name'] = data_name\n",
    "        df['Dataset ID'] = data_id\n",
    "        last_modified_time = int(os.path.getmtime(best_result_path))\n",
    "        last_modified_time = datetime.utcfromtimestamp(last_modified_time).strftime('%d-%m-%Y %H:%M')\n",
    "        df['Timestamp'] = last_modified_time\n",
    "        \n",
    "        results.append(df)\n",
    "\n",
    "    results = pd.concat(results, axis=0).reset_index(drop=True)\n",
    "    results = results[[\n",
    "        'Model name', 'Dataset ID', 'Dataset name', \n",
    "        'Adj. Best F1', 'rAUCROC', 'rAUCPR', 'VUSROC', \n",
    "        'VUSPR', 'Run name', 'ID', 'Timestamp']]\n",
    "    \n",
    "    return results, datasets_without_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "anomaly_transformer_results, datasets_without_results_at = get_model_results(model_name='AnomalyTransformer')\n",
    "moment_0_results, datasets_without_results_moment = get_model_results(model_name='MOMENT_0')\n",
    "moment_LP_results, datasets_without_results_moment = get_model_results(model_name='MOMENT_LP')\n",
    "dghl_results, datasets_without_results_dghl = get_model_results(model_name='DGHL')\n",
    "timesnet_results, datasets_without_results_timesnet = get_model_results(model_name='TimesNet')\n",
    "gpt4ts_results, datasets_without_results_gpt4ts = get_model_results(model_name='GPT4TS')\n",
    "ann_results, datasets_without_results_ann = get_model_results(model_name='AnomalyNearestNeighbors')\n",
    "\n",
    "results = pd.concat([moment_0_results, moment_LP_results, gpt4ts_results, timesnet_results, anomaly_transformer_results, dghl_results, ann_results], axis=0)\n",
    "results.to_csv(\"../../assets/results/finetuning/anomaly_detection.csv\", index=False)\n",
    "\n",
    "print(f\"\\n===Results shape===\")\n",
    "print(f\"MOMENT (zero-shot): {moment_0_results.shape}\")\n",
    "print(f\"MOMENT (linear probing): {moment_LP_results.shape}\")\n",
    "print(f\"AnomalyTransformer: {anomaly_transformer_results.shape}\")\n",
    "print(f\"DGHL: {dghl_results.shape}\")\n",
    "print(f\"TimesNet: {timesnet_results.shape}\")\n",
    "print(f\"GPT4TS: {gpt4ts_results.shape}\")\n",
    "print(f\"AnomalyNearestNeighbors: {ann_results.shape}\")\n",
    "print(f\"Total: {results.shape}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.dropna(inplace=True)\n",
    "print(f\"Results: {results.shape}\" )\n",
    "results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = results[['Model name', 'Dataset name', 'Adj. Best F1', 'VUSROC']]\n",
    "r[['Adj. Best F1']] = r[['Adj. Best F1']].apply(pd.to_numeric, errors='coerce')\n",
    "table = r.pivot_table(index='Dataset name', columns='Model name', values=['Adj. Best F1'], aggfunc='mean')\n",
    "average_rank = table.rank(axis=1, method='average', ascending=False)\n",
    "\n",
    "average_rank.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = results[['Model name', 'Dataset name', 'Adj. Best F1', 'VUSROC']]\n",
    "r[['VUSROC']] = r[['VUSROC']].apply(pd.to_numeric, errors='coerce')\n",
    "table = r.pivot_table(index='Dataset name', columns='Model name', values=['VUSROC'], aggfunc='mean')\n",
    "average_rank = table.rank(axis=1, method='average', ascending=False)\n",
    "\n",
    "average_rank.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Results for the paper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = results[['Model name', 'Dataset name', 'Adj. Best F1', 'VUSROC']]\n",
    "r[['Adj. Best F1', 'VUSROC']] = r[['Adj. Best F1', 'VUSROC']].apply(pd.to_numeric, errors='coerce')\n",
    "table = r.pivot_table(index='Dataset name', columns='Model name', values=['Adj. Best F1', 'VUSROC'], aggfunc='mean')\n",
    "table = table.round(2)\n",
    "table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "table.to_latex(\"../../assets/results/zero_shot/anomaly_detection.tex\", \n",
    "               multicolumn=True, multirow=True, multicolumn_format='c', float_format=\"%.3f\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'AnomalyNearestNeighbors' # MOMENT, AnomalyTransformer, DGHL, TimesNet, GPT4TS, AnomalyNearestNeighbors\n",
    "metrics = ['Adj. Best F1', 'rAUCROC', 'rAUCPR', 'VUSROC', 'VUSPR']\n",
    "results[results['Model name'] == model_name][metrics].astype(float).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'MOMENT' # MOMENT, AnomalyTransformer, DGHL\n",
    "results[results['Model name'] == model_name][metrics].astype(float).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'DGHL' # MOMENT, AnomalyTransformer, DGHL\n",
    "results[results['Model name'] == model_name][metrics].astype(float).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def box_plot(metric_name: str='Adj. Best F1'):\n",
    "    data = [\n",
    "    results[results['Model name'] == 'MOMENT'][metric_name].astype(np.float32),\n",
    "    results[results['Model name'] == 'AnomalyTransformer'][metric_name].astype(np.float32),\n",
    "    results[results['Model name'] == 'DGHL'][metric_name].astype(np.float32)]\n",
    "    \n",
    "    labels = ['MOMENT', 'Anomaly Transformer', 'DGHL']\n",
    "\n",
    "    boxprops = dict(linestyle='-', linewidth=1, color='k')\n",
    "    flierprops = dict(marker='o', markersize=12, markeredgecolor='darkgreen')\n",
    "    medianprops = dict(linestyle='-', linewidth=2, color='blue')\n",
    "    meanpointprops = dict(marker='D', markeredgecolor='black',\n",
    "                        markerfacecolor='firebrick')\n",
    "    meanlineprops = dict(linestyle='--', linewidth=2, color='red')\n",
    "\n",
    "    _ = plt.boxplot(\n",
    "        x=data, labels=labels, showmeans=True, \n",
    "        meanline=True, bootstrap=10000, notch=True, \n",
    "        boxprops=boxprops, flierprops=flierprops, \n",
    "        medianprops=medianprops, \n",
    "        meanprops=meanlineprops)\n",
    "    \n",
    "    if metric_name == 'Adj. Best F1':\n",
    "        plt.ylabel(\"Adjusted Best $F_1$\", fontsize=16)\n",
    "    else:\n",
    "        plt.ylabel(metric_name, fontsize=16)\n",
    "    \n",
    "    plt.xlabel(\"Models\", fontsize=16)\n",
    "    plt.ylim(0.0, None)\n",
    "    plt.xticks(fontsize=14)\n",
    "    plt.yticks(fontsize=14)\n",
    "    plt.grid(color='lightgray', linestyle='--', linewidth=0.5) \n",
    "    plt.savefig(f\"../../assets/figures/finetuning/anomaly_detection_{metric_name}.png\", \n",
    "                dpi=300, bbox_inches='tight')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(metric_name='Adj. Best F1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "box_plot(metric_name='VUSROC')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_without_results_at"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_without_results_dghl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets_without_results_moment"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Experiment scripts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "script = \"\"\n",
    "config = \"../../configs/anomaly_detection/linear_probing.yaml\"\n",
    "gpu_id = 0\n",
    "finetuning_mode = 'linear-probing'\n",
    "\n",
    "for dataset_name in ucr_anomaly_archive_subset:\n",
    "    script += f\"python anomaly_detection.py --config {config} --gpu_id {gpu_id} --finetuning_mode {finetuning_mode} --dataset_names {dataset_name}\\n\"\n",
    "\n",
    "print(script)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
